package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"strings"
	"sync"
)

var (
	domain    = "https://www.shopify.com"   // 域名
	linkFmt   = "%s/zh/blog/latest?page=%d" // 链接格式
	startPage = 1                           // 开始页
	endPage   = 45                          // 结束页
	saveDir   = "articles"                  // 保存文件目录
	fileExt   = "html"                      // 文件后缀
)

type Article struct {
	filename string
	link     string
}

var (
	pageWg      sync.WaitGroup
	articleWg   sync.WaitGroup
	pageChan    chan int     // 页面chan
	articleChan chan Article // 文章chan
)

func main() {
	fmt.Println("begin...")

	pageChan = make(chan int, 5)
	articleChan = make(chan Article, 5)

	// 开启多个协程出来
	for i := 0; i < 10; i++ {
		// go work(pageChan, articleChan)
		go pageWork(pageChan)
		go articleWork(articleChan)
	}

	for i := startPage; i <= endPage; i++ {
		pageChan <- i
		pageWg.Add(1)
	}

	pageWg.Wait()
	articleWg.Wait()

	fmt.Println("finish...")
}

func pageWork(ch chan int) {
	for {
		select {
		case pageNum := <-ch:
			link := fmt.Sprintf(linkFmt, domain, pageNum)
			content, err := getHtmlContent(link)
			if err == nil {
				parseHtml(content)
				pageWg.Done()
			} else {
				fmt.Println("[getHtmlContent] ", pageNum, err)
			}
		}
	}
}

func articleWork(ch chan Article) {
	for {
		select {
		case article := <-ch:
			saveToFile(article.filename, article.link)
			articleWg.Done()
		}
	}
}

// 获取html内容
func getHtmlContent(link string) (string, error) {
	resp, err := http.Get(link)
	if err != nil {
		fmt.Println("[http.Get] ", err)
		return "", err
	}

	defer resp.Body.Close()

	b, err := io.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("[io.ReadAll] ", err)
		return "", err
	}

	return string(b), nil
}

// 解析html
func parseHtml(content string) {
	// <a class="tracking-[-.02em] font-normal pb-4 hover:underline font-aktivgroteskextended text-shade-100 text-base" href="/zh/blog/monetizing-your-content" rel="" target="_self">Wait But Why创始人专访：受众变现的正确方式</a>
	// <a class="tracking-[-.02em] font-normal pb-4 hover:underline font-aktivgroteskextended text-shade-100 text-base" href="/zh/blog/business-marketing" rel="" target="_self">商业营销：类型、策略和示例</a>
	re := regexp.MustCompile(`<a class="tracking-\[-.02em\][a-zA-Z0-9\-\.: ]+" href="([a-zA-Z0-9\-/]+)" rel="" target="_self">([^>]+)</a>`)
	result := re.FindAllStringSubmatch(content, -1)

	articleWg.Add(len(result))
	for _, v := range result {

		filename := strings.ReplaceAll(v[2], "?", "？")
		filename = strings.ReplaceAll(filename, "/", "")
		articleChan <- Article{
			filename: filename,
			link:     fmt.Sprintf("%s%s", domain, v[1]),
		}
	}
}

// 保存到文件
func saveToFile(filename string, link string) {
	content, err := getHtmlContent(link)
	if err != nil {
		fmt.Println("[getHtmlContent] ", link, err)
		return
	}

	path := fmt.Sprintf("%s/%s.%s", saveDir, filename, fileExt)
	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Println("[os.OpenFile] ", link, err)
		return
	}

	defer f.Close()

	f.WriteString(content)
}
